package com.icarbonx.dap.notification.util

import org.apache.spark.SparkConf

/**
 * Created by Lenovo on 2018/8/23.
 */
object SparkUtil {

  def getSparkConf(appName: String, classes: Array[Class[_]], maxRate: String = "100") = {
    val conf: SparkConf = new SparkConf()
    //    conf.set("spark.akka.frameSize", "2047")
    //    conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") //一些默认的类使用kryo序列化
    //    conf.set("spark.kryoserializer.buffer.max.mb", "2040")
    //    conf.set("spark.files.overwrite", "true")
    //    conf.set("spark.hadoop.validateOutputSpecs", "false")
    //    conf.set("spark.eventLog.overwrite", "true")
    conf.set("spark.streaming.kafka.maxRatePerPartition", maxRate) //每秒钟最大消费，而kafka拉的数据为topic对应partition的数量乘以设置的数
    conf.setAppName(appName)
    conf.registerKryoClasses(classes)
//    conf.setMaster("local[3]")
  }

}
